package com.gome.ss.flowcrawl.plugin;

import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeoutException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;

import com.gome.spiderkit.client.Cookie;
import com.gome.spiderkit.client.CookieJar;
import com.gome.spiderkit.client.Rect;
import com.gome.spiderkit.client.UnknowServerException;
import com.gome.spiderkit.client.Utils;
import com.gome.spiderkit.client.WebElement;
import com.gome.spiderkit.client.WebPage;
import com.gome.ss.core.download.SpiderDataMap;
import com.gome.ss.core.download.SpiderRequest;
import com.gome.ss.core.download.SpiderRequestInterface;
import com.gome.ss.core.download.SpiderResponseInterface;
import com.gome.ss.core.plugin.SpiderLocalPluginActionConfig;
import com.gome.ss.core.plugin.SpiderLocalPluginStepConfig;
import com.gome.ss.core.queue.QueueCache;
import com.google.protobuf.InvalidProtocolBufferException;

public class FlowCrawlSpiderCommon {
	private static Logger logger = Logger.getLogger(FlowCrawlSpiderCommon.class);
	
	private static void doAction(SpiderLocalPluginActionConfig ac, WebPage page, WebElement element) throws Exception {
		for (int i = 0; i < ac.getRepTimes(); i ++) {
			if (ac.isClick()) {
				element.click();
			} else if (ac.isScroll()) {
				Rect r = element.getRect();
				page.scroll(r.getX(), r.getY());
			} else if (ac.isSenkey()) {
				element.sendKeys("keypress", ac.getKvalue());
			} else if (ac.isType()) {
				element.type(ac.getTvalue());
			}
			Thread.sleep(ac.getSleepTime());
		}
	}
	
	public static void onDownLoadFinished(SpiderResponseInterface response, SpiderLocalPluginStepConfig config)
			throws Exception {
		WebPage page = response.loadedPage();
		SpiderDataMap dataMap = new SpiderDataMap();
		
		dataMap.add("failedWait", "0");
		dataMap.add("failedAction", "0");
		
		boolean hasDoneAOverlapAction = false;
		
		List<SpiderLocalPluginActionConfig> bactions = config.getBaction();
		for (SpiderLocalPluginActionConfig ac : bactions) {
			if (hasDoneAOverlapAction && ac.isOverlap())
				continue;
			
			if (!hasDoneAOverlapAction && ac.isOverlap() && ac.isHasDone())
				continue;
			
			if (!hasDoneAOverlapAction && ac.isOverlap() && !ac.isHasDone()) {
				hasDoneAOverlapAction = true;
				ac.setHasDone(true);
				String keyWord = ac.getTvalue();

				if (ac.isIntoKeyword() && keyWord != null && !keyWord.equals(""))
					dataMap.add("actionKeyWord", keyWord);
			}
			
			if (page.waitForWebElement(ac.getCss(), 1000)) {
				if (ac.isIsmultiple()) {
					List<WebElement> elements = page.getElementsByCssSelector(ac.getCss());
					for (WebElement element : elements) {
						doAction(ac, page, element);
						Thread.sleep(50);
					}
				} else {
					WebElement element = page.getElementByCssSelector(ac.getCss());
					doAction(ac, page, element);
				}
			} else {
				dataMap.add("failedAction", "1");
				response.needRetry();
			}
		}
		
		List<String> failedWaitCss = new ArrayList<String>();
		List<String> waitCsses = config.getWaitCssList();
		for (String css : waitCsses) {
			if (!page.waitForWebElement(css, 1000)) {
				dataMap.add("failedWait", "1");
				response.needRetry();
			}
		}
		
		response.getDataMap().add("dataMap", dataMap);
	}
	
	public static List<SpiderRequestInterface> parseLink(
			SpiderResponseInterface response, SpiderLocalPluginStepConfig config) throws Exception {
		
		if (config.isEnd())
			return null;

		List<SpiderRequestInterface> requests = new ArrayList<SpiderRequestInterface>();
		
		WebPage page = response.loadedPage();
		String link = page.getUrl();
		CookieJar cookies = page.getCookieJar();
		
		Map<String, String> pageInfo = new HashMap<String, String>();
		
		SpiderDataMap dataMap = (SpiderDataMap) response.getDataMap().get("dataMap");
		if (dataMap != null && dataMap.get("actionKeyWord") != null) {
			String keyWord = (String) dataMap.get("actionKeyWord");
			if (!config.isEntry()) {
				Map<String, String> parentPage = (Map<String, String>)response.getData();
				if (parentPage != null) {
					String pkeyw = parentPage.get("actionKeyWord");
					keyWord = pkeyw + ";" + keyWord;
				}
			}
			if (!keyWord.equals(""))
				pageInfo.put("actionKeyWord", keyWord);
		} else {
			if (!config.isEntry()) {
				Map<String, String> parentPage = (Map<String, String>)response.getData();
				if (parentPage != null) {
					String keyWord = parentPage.get("actionKeyWord");
					if (keyWord != null && !keyWord.equals(""))
						pageInfo.put("actionKeyWord", keyWord);
				}
			}
		}
		
		pageInfo.put("id", response.getID());
		pageInfo.put("url", link);
		pageInfo.put("qn", config.getQualifiedName());
		pageInfo.put("stepID", String.valueOf(config.getStepID()));
		pageInfo.put("stepName", config.getStepName());
		pageInfo.put("cookies", cookies.toJson());
		
		CookieJar newCookies = null;
		List<Cookie> newCookieList = new ArrayList<Cookie>();
		if(config.isNeedCarryCookie()) {
			List<String> names = config.getCarryCookieName();
			if (names.size() == 1 && names.get(0).equals("*")) {
				newCookies = cookies;
			} else {
				for (String n : names) {
					if (n.equals(""))
						continue;
					
					Cookie c = cookies.get(n);
					if (c != null)
						newCookieList.add(c);
				}
				newCookies = new CookieJar(newCookieList);
			}
		} else {
			page.setCookies(new ArrayList<Cookie>());
		}
		
		if (config.getNextStepType().equals(SpiderLocalPluginStepConfig.HREFTYPE)) {
			String css = config.getNextStepCss();
			if (css != null && !css.equals("") && page.waitForWebElement(css, 100)) {
				List<WebElement> elements = page.getElementsByCssSelector(css);
				for (WebElement element : elements) {
					String href = element.getAttribute(config.getNextStepHrefAttribute());
					if (href != null && !href.equals("")) {
						Pattern p = Pattern.compile("^[a-zA-z]+://.*");
						Matcher m = p.matcher(href);
						if(!m.matches()) {
							URL url = new URL(link);
							if (href.startsWith("/")) {
								href = url.getProtocol() + "://" + url.getHost() + href;
							} else {
								if (href.startsWith("?"))
									href = url.getProtocol() + "://" + url.getHost() + url.getPath() + href;
								else
									href = url.getProtocol() + "://" + url.getHost() + url.getPath() + "/" + href;
							}
						}
						
						SpiderRequestInterface request = null;
						if (newCookies == null) {
							request = SpiderRequest.create(href, 
								config.getGroup(),
								config.getPluginName(),
								config.getNextStepQualifiedName(), 
								QueueCache.defaultPriority, 3);
							request.setIncremental(true);
						} else {
							request = SpiderRequest.create(href, 
									config.getGroup(),
									config.getPluginName(),
									config.getNextStepQualifiedName(),
									newCookies,
									QueueCache.defaultPriority, 3);
							request.setIncremental(true);
						}
						request.carryData(pageInfo);
						requests.add(request);
					}
				}
			}
		} else if (config.getNextStepType().equals(SpiderLocalPluginStepConfig.CLICKTYPE)) {
			String css = config.getNextStepCss();
			if (page.waitForWebElement(css, 100)) {
				List<WebElement> elements = page.getElementsByCssSelector(css);
				if (config.isNextStepOpenNewPage()) {
					for (int i = 0; i < elements.size(); i ++) {
						WebElement element = elements.get(i);
						WebPage newPage = new WebPage();
						if (page.click(element, true, newPage)) {
							if (!newPage.isFailed()) {
								SpiderRequestInterface request = SpiderRequest.createPageReady(newPage,
										true, // new page
										config.getPluginName(),
										config.getNextStepQualifiedName(), 1);
								request.carryData(pageInfo);
								requests.add(request);
							}
						} else {
							response.setRetry();
						}
					}
				} else {
					WebElement element = elements.get(0);
					if (page.click(element)) {
						SpiderRequestInterface request = SpiderRequest.createPageReady(page,
								config.getPluginName(),
								config.getNextStepQualifiedName(), 1);
						request.carryData(pageInfo);
						requests.add(request);
					} else {
						response.setRetry();
					}
					/* current page is changed here, can't continue parse if it's a list page!*/
					return requests;
				}
			}
		}
		
		if (!config.isList())
			return requests;
		
		Map<String, String> parentPageInfo = (Map<String, String>)response.getData();
		
		if (parentPageInfo == null)
			parentPageInfo = new HashMap<String,String>();
		
		String currentPageCount = "";
		WebElement cpElement = null;
		if (page.waitForWebElement(config.getCurrentPageCss(), 200)) {
			cpElement = page.getElementByCssSelector(config.getCurrentPageCss());
			currentPageCount = cpElement.getText();
			logger.info("Current page count : " + currentPageCount + " --- link : " + link);
		} else {
			logger.error("Can't found current page count element " + config.getCurrentPageCss() + " --- link : " + link);
			return requests;
		}
		
		if (config.getNextPagesSettingType().equals("0")) {
			String css = config.getPageCountCss();
			if (page.waitForWebElement(css, 100)) {
				WebElement element = page.getElementByCssSelector(css);
				
				String pcTotalString = element.getText();
				pcTotalString = pcTotalString.trim();
				
				logger.info("Total page count : " + pcTotalString + " --- link : " + link);
				
				try {
					int pcTotal = Integer.valueOf(pcTotalString);
					int pcStart = Integer.valueOf(config.getPageCountStartWith());
					
					for (int i = pcStart; i < pcTotal + pcStart; i ++) {
						URL url = new URL(link);
						String href = url.getProtocol() + "://" + url.getHost() +
									  url.getPath() +
									  config.getNextPageParameterPrefix() +
									  String.valueOf(i) +
									  config.getNextPageParameterSuffix();
						
						SpiderRequestInterface request = SpiderRequest.create(href, 
								config.getGroup(),
								config.getPluginName(),
								config.getQualifiedName(), 
								QueueCache.defaultPriority, 3);
						request.setListPage(true);
						request.setCurrentPage(currentPageCount);
						request.carryData(pageInfo);
						requests.add(request);
					}
				} catch (Exception e) {
					logger.error(Utils.getTrace(e));
				}
			} else {
				logger.error("Can't found page count element " + config.getPageCountCss());
			}
		} else if (config.getNextPagesSettingType().equals("1")) {
			String css = config.getNextPageCss();
			if (page.waitForWebElement(css, 100)) {
				List<WebElement> elements = page.getElementsByCssSelector(css);
				WebElement element = elements.get(0);
				if (elements.size() > 1) {
					for (WebElement e : elements) {
						String text = e.getText();
						if (text != null && text.indexOf("下") != -1 && text.indexOf("页") != -1) {
							element = e;
							break;
						}
					}
				}
				
				String classAttribute = element.getAttribute("class");
				if (classAttribute != null && classAttribute.indexOf("disable") != -1) {
					return requests;
				}
				
				if (config.getNextPageType().equals(SpiderLocalPluginStepConfig.HREFTYPE)) {
					String href = element.getAttribute(config.getNextPageHrefAttribute());
					if (href != null && !href.equals("")) {
						Pattern p = Pattern.compile("^[a-zA-z]+://.*");
						Matcher m = p.matcher(href);
						if(!m.matches()) {
							URL url = new URL(link);
							if (href.startsWith("/")) {
								href = url.getProtocol() + "://" + url.getHost() + href;
							} else {
								if (href.startsWith("?"))
									href = url.getProtocol() + "://" + url.getHost() + url.getPath() + href;
								else
									href = url.getProtocol() + "://" + url.getHost() + url.getPath() + "/" + href;
							}
						}
						SpiderRequestInterface request = SpiderRequest.create(href, 
								config.getGroup(),
								config.getPluginName(),
								config.getQualifiedName(), 
								QueueCache.defaultPriority, 3);
						request.setListPage(true);
						request.setCurrentPage(currentPageCount);
						request.carryData(pageInfo);
						requests.add(request);
					}
				} else if (config.getNextPageType().equals(SpiderLocalPluginStepConfig.CLICKTYPE)) {
					if (page.waitForWebElement(config.getNextPageCss(), 100)) {
						if (config.isNextPageASync()) {
							/*
							 * 关于重试考虑不周全，先放弃
							if (response.isRetry() && response.loadedPage() == page) {
								String needJumpPageCount = response.currentPage();
								int tryCount = Integer.valueOf(needJumpPageCount) * 3;
								while(!needJumpPageCount.equals(currentPageCount)) {
									element.click();
									Thread.sleep(config.getNextPageClickWaitTime()*1000);
									
									cpElement = page.getElementByCssSelector(config.getCurrentPageCss());
									currentPageCount = cpElement.getText();
									logger.info("Retry click : " + currentPageCount + " | " + config.getNextPageCss() + " --- link : " + link);
									tryCount --;
									if (tryCount < 0) {
										response.setRetry();
										logger.error("Retry click failed :" + config.getNextPageCss() + " --- link : " + link);
										return requests;
									}
								}
							}
							*/
							
							String currentCheck = "";
							if (page.waitForWebElement(config.getNextPageClickWaitCssChanged(), 10)) {
								currentCheck = page.getElementByCssSelector(config.getNextPageClickWaitCssChanged()).getText();
							}
							
							logger.info("Current page " + currentPageCount + " Click : " + config.getNextPageCss() + " check " + currentCheck + " --- link : " + link);
							
							String tempCheck = "";
							for (int j = 0; j < 3; j ++) {
								element.click();
								Thread.sleep(config.getNextPageClickWaitTime()*1000);
								
								if (config.getNextPageClickWaitCssChanged().equals(""))
									break;
								
								WebElement checkElement = page.getElementByCssSelector(config.getNextPageClickWaitCssChanged());
								tempCheck = checkElement.getText();
								if (!currentCheck.equals(tempCheck))
									break;
							}
							
							logger.info("Click check info : " + currentCheck + " | " + tempCheck);
							if (currentCheck.equals(tempCheck)) {
								logger.error("Click to next page error : " + config.getNextPageCss() + " --- link : " + link);
								return requests;
							} else {
								SpiderRequestInterface request = SpiderRequest.createPageReady(page,
										config.getPluginName(),
										config.getQualifiedName(), 1);
								request.carryData(pageInfo);
								requests.add(request);
							}
						} else {
							if (page.click(element)) {
								SpiderRequestInterface request = SpiderRequest.createPageReady(page,
										config.getPluginName(),
										config.getQualifiedName(), 1);
								request.setListPage(true);
								request.setCurrentPage(currentPageCount);
								request.carryData(pageInfo);
								requests.add(request);
							} else {
								response.setRetry();
							}
						}
					}
				}
			}
		}
		
		return requests;
	}

	public static SpiderDataMap parseData(SpiderResponseInterface response, SpiderLocalPluginStepConfig config)
			throws Exception {
		
		WebPage page = response.loadedPage();
		
		SpiderDataMap dataMap = (SpiderDataMap) response.getDataMap().get("dataMap");
		
		String pkeyWord = "";
		if (!config.isEntry()) {
			Map<String, String> parentPage = (Map<String, String>)response.getData();
			if (parentPage != null) {
				if (parentPage.get("actionKeyWord") != null)
					pkeyWord = parentPage.get("actionKeyWord");
				dataMap.add("parent", parentPage);
			}
		}
		
		CookieJar cookies = page.getCookieJar();
		
		try {
			String title = page.getTitle();
			
			dataMap.add("id", response.getID());
			dataMap.add("stepID", config.getStepID());
			dataMap.add("date", (new SimpleDateFormat("yyyy-MM-dd HH:mm:ss:SSS")).format(new Date()));
			if (title != null)
				dataMap.add("title", title);
			else
				dataMap.add("title", "null");
			dataMap.add("url", page.getUrl());
			dataMap.add("cookies", cookies.toJson());
			dataMap.add("pageSource", page.getPageSource());
			dataMap.add("crawlName", config.getPluginName());
			
			String keyWord = "";
			
			if (config.getKeyWord() != null) {
				String ckeyWord = config.getKeyWord();
				if (!ckeyWord.equals("")) {
					keyWord = ckeyWord;
					if (!pkeyWord.equals(""))
						keyWord = ckeyWord + ";" + pkeyWord;
				} else if (!pkeyWord.equals(""))
					keyWord = pkeyWord;
			}
			
			dataMap.add("keyWord", keyWord);
		} catch (NullPointerException e) {
			if (page.isFailed()) {
				logger.error("Page error!");
			} else {
				logger.error(Utils.getTrace(e));
				throw e;
			}
		}
		
		return dataMap;
	}
	
}
